# -*- coding: utf-8 -*-
# time: 2025/5/14 13:49
# file: split01.py
# author: hanson
"""
https://www.langchain.com.cn/docs/how_to/semantic-chunker/
按标题分割Markdown
"""
from langchain_community.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings
from langchain_experimental.text_splitter import SemanticChunker
from langchain_ollama import OllamaEmbeddings

file = r"F:\workspace\py_project\intellect\llm\data\tsData.txt"

# Load example document
with open(file, encoding="utf-8") as f:
    state_of_the_union = f.read()


# text_splitter = SemanticChunker(OpenAIEmbeddings())
# 使用 HuggingFace Embedding
#embedding = HuggingFaceEmbeddings(model_name="BAAI/bge-small-zh")  # 中文
#embedding = HuggingFaceEmbeddings(model_name='E:/opt/AI/text2vec-base-chinese')

# 使用 Ollama 本地 Embedding
embedding = OllamaEmbeddings(model="nomic-embed-text")
text_splitter = SemanticChunker(embedding)
# embedding = HuggingFaceEmbeddings(model_name="BAAI/bge-small-en")  # 英文

docs = text_splitter.create_documents([state_of_the_union])
print(docs[0].page_content)
print("---------------------------")
print(docs[1].page_content)


